{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoder(nn.Module):\n",
    "    def __init__(self,d_model,max_seq_len=80):\n",
    "        super.__init__()\n",
    "        self.d_model=d_model\n",
    "\n",
    "        pe=torch.zeros(max_seq_len,d_model)\n",
    "        for pos in range(max_seq_len):\n",
    "            for i in range(0,d_model,2):\n",
    "                pe[pos,i]=math.sin(pos/(10000**((2*i)/d_model)))\n",
    "                pe[pos,i+1]=math.cos(pos/(1000**((2*(i+1))/d_model)))\n",
    "        \n",
    "        pe=pe.unsqueeze(0)\n",
    "        self.register_buffer('pe',pe)\n",
    "    def forward(self,x):\n",
    "        x=x*math.sqrt(self.d_model)\n",
    "\n",
    "        seq_len=x.size(1)\n",
    "        x=x+Variable(self.pe[:,:seq_len],requires_grad=False).cuda()#????\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.8012, 0.3071]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i=torch.randn((1,2))\n",
    "i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8167, 0.1833]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.softmax(i,dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self,heads,d_model,dropout=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.d_model=d_model\n",
    "        self.d_k=d_model//heads\n",
    "        self.h=heads\n",
    "\n",
    "        self.q_linear = nn.Linear(d_model, d_model)\n",
    "        self.v_linear = nn.Linear(d_model, d_model)\n",
    "        self.k_linear = nn.Linear(d_model, d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.out = nn.Linear(d_model, d_model)\n",
    "    def attention(q,k,v,d_k,mask=None,dropout=None):\n",
    "        scores=torch.matmul(q,k.transpose(-2,-1))/math.sqrt(d_k)\n",
    "\n",
    "        if mask is not None:\n",
    "            mask=mask.unsqueeze(1)\n",
    "            scores=scores.masked_fill(mask==0,-1e9)\n",
    "\n",
    "        scores=F.softmax(scores,dim=-1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "LLM",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
